﻿################3最基本的抓站
from urllib import request

init_url = "http://www.baidu.com"
with request.urlopen(init_url) as f:
	pass
	# print(f.read())

#############使用代理服务器
#ip被封了，ip被访问的次数过多
# proxy_support = request.ProxyHandler({'http':'http://xx.xx.xx.xx:xx'})  
# opener = request.build_opener(proxy_support, request.HTTPHandler)  
# request.install_opener(opener)  
  
# content = request.urlopen('http://www.baidu.com/').read().decode('utf-8')  
# print(content) 

########需要登陆的情况#########
#####cookie的处理
from http import cookiejar

cookie_support = request.HTTPCookieProcessor(cookiejar.CookieJar())
opener = request.build_opener(cookie_support, request.HTTPHandler) 
request.install_opener(opener)  
  
content = request.urlopen('http://www.baidu.com/').read().decode('utf-8')  
print(content)  

# #####表单的填写
# postdata = parse.urlencode({  
    # 'username':'XXXXX',  
    # 'password':'XXXXX',  
    # 'continueURI':'http://www.verycd.com/',  
    # 'fk':'fkasdfasdf',  
    # 'login_submit':'登录',  
# })  
# req = request.Request(  
    # url = 'http://secure.verycd.com/signin/*/http://www.verycd.com/',  
    # data = postdata  
# )  
# content = request.urlopen(req).read()  
  
# print(content)  
# ###get请求
# fullurl = 'http://xxxxxxxxxxxxxxx/index?%s' % data  
# req = request.Request(fullurl)  

########伪装成浏览器访问
# postdata = parse.urlencode({})  
# headers = {  
    # 'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'  
# }  
# req = request.Request(  
    # url = 'http://secure.verycd.com/signin/*/http://www.verycd.com/',  
    # data = postdata,  
    # headers = headers  
# )

######反盗链
headers = {  
    'Referer':'http://www.cnbeta.com/articles'  
}  

################多线程并发抓取###############
from threading import Thread  
from queue import Queue  
from time import sleep  
#q是任务队列  
#NUM是并发线程总数  
#JOBS是有多少任务  
q = Queue()  
NUM = 2  
JOBS = 10  
#具体的处理函数，负责处理单个任务  
def do_somthing_using(arguments):  
    print(arguments)  
      
#这个是工作进程，负责不断从队列取数据并处理  
def working():  
    while True:  
        arguments = q.get()  
        do_somthing_using(arguments)  
        sleep(1)  
        q.task_done()  
#fork NUM个线程等待队列  
for i in range(NUM):  
    t = Thread(target=working)  
    t.setDaemon(True)  
    t.start()  
#把JOBS排入队列  
for i in range(JOBS):  
    q.put(i)  
#等待所有JOBS完成  
q.join()  